////////////////// SET UP YOUR DATA EXTRACTION  - BEGIN //////////////////////////////////
categorizeBasedOnPath = { ln -> 
                if(ln ==~ /.*GET \/\".*/) { return "ROOT_REQUEST" }
                        if(ln ==~ /.*GET \/ HTTP\/1[.][01].*/) { return "ROOT_REQUEST" }
                        if(ln ==~ /.*GET \/services\/MAM.*/) { return "MAM_REQUEST" }
                        if(ln ==~ /.*GET \/app\/crm.*/) { return "APP_REQUEST" }
                        if(ln ==~ /.* \/jasmine\/tnc.html.*/) { return "APP_REQUEST" }
                        if(ln ==~ /.* \/jasmine\/privacy.html.*/) { return "APP_REQUEST" }
                        if(ln ==~ /.*POST \/app\/crm.*/) { return "LOGIN_LOGOUT_REQUEST" }
                        if(ln ==~ /.*GET \/favicon[.]ico.*/) { return "APP_REQUEST" }
                        if(ln ==~ /.*GET \/services\/identity.*/) { return "IDENTITY_SERVICE_REQUEST_GET" }
                        if(ln ==~ /.*POST \/services\/identity.*/) { return "IDENTITY_SERVICE_REQUEST_POST" }
                        if(ln ==~ /.*GET \/services.*/) { return "SERVICE_REQUEST_GET" }
                        if(ln ==~ /.*POST \/services.*/) { return "SERVICE_REQUEST_POST" }
                        if(ln ==~ /.*POST \/jasmine\/mopush.*/) { return "JASMINE_MOPUSH" }
                        if(ln ==~ /.*GET \/jasmine\/mopush.*/) { return "JASMINE_MOPUSH" }
                        if(ln ==~ /.*GET \/robots[.]txt.*/) { return "ROBOTS" }
                        if(ln ==~ /.*HEAD .*/ ||
                           ln ==~ /.*OPTIONS .*/ || 
                           ln ==~ /.*TRACE .*/ ||
                           ln ==~ /.*PUT .*/ ||
                           ln ==~ /.*DELETE .*/ ) { return "UNSUPPORTED_METHOD_REQUEST" }
                        return "UNKNOWN_REQUEST"
        }
getHttpStatusCode = { ln->
                def m = ln =~ /\"[^\"]*\" ([0-9]+) .*/
                if(!m || m.groupCount() < 1)return false;
                return m.group(1)
        }
getBytesReturned = { ln->
                def m = ln =~ /\"[^\"]*\" [0-9]+ ([0-9]+).*/
                if(!m || m.groupCount() < 1)return false;
                return m.group(1)
        }
getRealm = {ln ->
               def m =ln =~ /.*[PG][OE]S?T (\/services\/[^ ]*\/)([0-9]+)[\/][0-9]* HTTP.*/
               if(!m || m.groupCount() < 2)return false;
               if(m.group(1) =~ /.*services\/identity\/preference\/user.*/) return false //This is authid
               m.group(2)
        }

membershipFunction = {ln -> return (categorizeBasedOnPath(ln).startsWith("SERVICE_REQUEST"))}
////////////////// SET UP YOUR DATA EXTRACTION  - END //////////////////////////////////

def window = 60000 //milliseconds; for moving average
evaluate( new File('Count.groovy'))
evaluate( new File('MovingAverageLoadUnsortedLog.groovy'))

////////////////// SET UP YOUR PIPELINE  - BEGIN //////////////////////////////////
pipeline = { ln ->
                 count(ln, "Realms", getRealm)
                 count(ln, "Counts", categorizeBasedOnPath)
                 //count(ln, "HttpStatusCodes", getHttpStatusCode)
                 //accumulate(ln, "BytesReturned", getBytesReturned)
                 //printUnknownRequests(ln, categorizeBasedOnPath)
                 movingAverage(ln, window, "services", membershipFunction)
}

resultsPipeline = {
        countResult("Realms").each{ println it.key +': '+ it.value }
        countResult("Counts").each{ println it.key +': '+ it.value }
        //countResult("HttpStatusCodes").each{ println it.key +': '+ it.value }
        //println accumulateResult("BytesReturned")
        def mvAvg = movingAverageResult("services", window)
        println "Max of moving average for services : "+mvAvg.max()
        println "Min of moving average for services : "+mvAvg.min()
}
////////////////// SET UP YOUR PIPELINE  - END //////////////////////////////////


directory = 'D:\\jasmine_admin\\jasmine_prod_webserver_logs'
patternOfFilesToParse = ~/.*\.access_log.*/
evaluate( new File('Parse.groovy'))
resultsPipeline()


//TODO: find popular useragent
//TODO: auto find log patterns
//TODO: make it mapped/reduced
